//==============================================================================================================================================================
// Author(s): Roman Khmelichek
//
// Implements the callback functions for the parser.
//==============================================================================================================================================================

#ifndef PARSER_CALLBACK_H_
#define PARSER_CALLBACK_H_

// Enables debugging output for this module.
// #define PARSER_CALLBACK_DEBUG

#include <boost/algorithm/string.hpp>
#include <fstream>
#include <iostream>
#include <map>
#include <string>
#include <strings.h>
#include <utility>

#include "config_file_properties.h"
#include "configuration.h"
#include "globals.h"
#include "logger.h"
#include "posting_collection.h"

using namespace boost;
using namespace std;
/**************************************************************************************************************************************************************
 * ParserCallback
 *
 **************************************************************************************************************************************************************/
class ParserCallback {
public:
  void ProcessTerm(const char* term, int term_len, uint32_t doc_id, uint32_t position, unsigned char context, bool weiSwitch){
  }

  // URLs could begin with "http://".
  // Might want to strip "#" from URLs.
  void ProcessUrl(const char* url, int url_len, uint32_t doc_id) {
  }

  // Indicates the start of a new document.
  // The TREC DOCNO specifies the document bundle folder, bundle file, and the document's byte offset within the uncompressed bundle file.
  void ProcessDocno(const char* docno, int docno_len, uint32_t doc_id) {
  }

  void ProcessDocLength(int doc_length, uint32_t doc_id) {
  }

  // Out links might need to be concatenated with the base URL (if they're relative URLs).
  void ProcessLink(const char* url, int url_len, uint32_t doc_id) {
  }

};

/**************************************************************************************************************************************************************
 * IndexingParserCallback
 *
 **************************************************************************************************************************************************************/
class IndexingParserCallback : public ParserCallback {
public:
  IndexingParserCallback(PostingCollectionController* posting_collection_controller,EdgeCollectionController* edge_collection_controller);
  IndexingParserCallback(PostingCollectionController* posting_collection_controller);

  ~IndexingParserCallback();

  // Updated by Wei 2013/03/13
  // The weiSwitch controls whether runs the logic of the phase2 pruning.
  // Usually, the switch is set to be false
  void ProcessTerm(const char* term, int term_len, uint32_t doc_id, uint32_t position, unsigned char context, bool weiSwitch);

  void ProcessDocLength(int doc_length, uint32_t doc_id);

  void ProcessUrl(const char* url, int url_len, uint32_t doc_id);

  void ProcessDocno(const char* docno, int docno_len, uint32_t doc_id);

  EdgeCollectionController* GetEdgeCollectionController();

private:
  PostingCollectionController* posting_collection_controller_;
  EdgeCollectionController* edge_collection_controller_;

};

inline EdgeCollectionController* IndexingParserCallback::GetEdgeCollectionController(){
	return edge_collection_controller_;
}

inline IndexingParserCallback::IndexingParserCallback(PostingCollectionController* posting_collection_controller,EdgeCollectionController* edge_collection_controller) :
  posting_collection_controller_(posting_collection_controller),
  edge_collection_controller_(edge_collection_controller){
}


inline IndexingParserCallback::IndexingParserCallback(PostingCollectionController* posting_collection_controller) :
  posting_collection_controller_(posting_collection_controller),
  edge_collection_controller_(NULL){
}

inline IndexingParserCallback::~IndexingParserCallback(){
	// for debug ONLY.
	// maybe this is the problem
	// outputFileHandlerForDocumentEdges_.close();
}

inline void IndexingParserCallback::ProcessTerm(const char* term, int term_len, uint32_t doc_id, uint32_t position, unsigned char context, bool weiSwitch) {
    // cout << "the function IndexingParserCallback::ProcessTerm(...) called." << endl;
	static uint32_t prev_doc_id = 0;
#ifdef PARSER_CALLBACK_DEBUG
  // Detects skips in docIDs. Assumes docIDs assigned sequentially. For catching potential parser bugs.
  //std::cout << "IndexingParserCallback::ProcessTerm called()." << std::endl;
  //std::cout << "The term that is dealing with is:"<< term << std::endl;
  static int lost_doc_id_count = 0;
  if (doc_id > prev_doc_id) {
    if (doc_id > (prev_doc_id + 1)) {
      GetErrorLogger().Log("No postings for docID: " + Stringify(prev_doc_id + 1) + " and " + Stringify(doc_id - prev_doc_id - 2) + " more docs.", false);
      lost_doc_id_count += (doc_id - prev_doc_id - 1);
    }
    prev_doc_id = doc_id;
  }
#endif

  string termInStringFormat = string(term,term_len);
  // The following logic is to normalize the words extracted from the webpage
  for (size_t i = 0; i < termInStringFormat.size(); i++) {
	// normalization operation1
	if (isupper(termInStringFormat[i])){
		termInStringFormat[i] = tolower(termInStringFormat[i]);
	}

	/*
	// normalization operation2
	// We need to remove punctuation, since we only index alphanumeric characters and anything separated by a non-alphanumeric
	// character is considered a token separator by our parser. Not removing punctuation will result in the token not being found in the lexicon.
	int int_val = termInStringFormat[i];
	if (!((int_val >= 48 && int_val < 58) || (int_val >= 65 && int_val < 91) || (int_val >= 97 && int_val < 123) || (int_val == 32))) {
		termInStringFormat[i] = ' ';  // Replace it with a space.
	}
	*/
  }

  edge_collection_controller_->InsertTermIntoCorrespondingTermDicts(termInStringFormat, doc_id);

  // Updated by Wei 2013/07/15, this logic is corresponding with another function called: Parser<Callback>::ParseTrecDataBufferForPhase2Pruning
  // If I am using that function, I will un_comment the following statements:
  // But if I am using another function called:Parser<Callback>::ParseTrecDataBufferForPhase2PruningNewTryUpdatedOn20130714Night, I will comment the following statements
  /*
  if(doc_id > prev_doc_id){
	  if(weiSwitch){
		  // cout << "Do the document posting graph for the doc_id:" << prev_doc_id << endl;

		  // for debug ONLY
		  // cout << "doc_id_with_terms_dict_.size() before:" << doc_id_with_terms_dict_.size() << endl;
		  // cout << "doc_id_with_terms_dict_[prev_doc_id].size() before:" << doc_id_with_terms_dict_[prev_doc_id].size() << endl;

		  // option1: run the method1
		  // edge_collection_controller_->GenerateDocumentPostingGraphMethod1( prev_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );

		  // option2: run the method2
		  // edge_collection_controller_->GenerateDocumentPostingGraphMethod2( prev_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );

		  // option3: run the method3
		  // edge_collection_controller_->GenerateDocumentPostingGraphMethod3( prev_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );

		  // option4: run the method1 and method3 together with the setting of the threshold.
		  // cout << "edge_collection_controller_->GetProperThreshold():" << edge_collection_controller_->GetProperThreshold() << endl;
		  // for comparison
		  // edge_collection_controller_->GenerateDocumentPostingGraphMethod1( prev_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );
		  // edge_collection_controller_->GenerateDocumentPostingGraphMethod3( prev_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)) );


		  // ***********************************current used version since 2013/07/09*******************************
		  // Logic begins...
		  // Block the following function for parsing speedup (for Shi's experiment)

		  if (edge_collection_controller_->GetProperThreshold() <= 180){
			  edge_collection_controller_->GenerateDocumentPostingGraphMethod3( prev_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)), false, false );
		  }
		  else{
			  edge_collection_controller_->GenerateDocumentPostingGraphMethod1( prev_doc_id, Configuration::GetResultValue<long int>(Configuration::GetConfiguration().GetNumericalValue(config_properties::kTermPairEdgeFreqThreshold)), false, false );
		  }

		  // Logic ends.
		  // ***********************************current used version since 2013/07/09*******************************

	  }
	  else{
		  // should clear the variable for the current document
		  // even NOT doing the graph generation part, you still need to clear up the data structures.
		  edge_collection_controller_->ClearDocIDRelatedTermDicts(prev_doc_id); // erasing the whole doc by key
		  edge_collection_controller_->ClearTermsBothInCurrentDocIDAndQueryTraceVector(); // clear the terms_both_in_current_doc_id_and_query_trace_vector_
	  }
  }
  */

  prev_doc_id = doc_id;

  Posting posting(term, term_len, doc_id, position, context);
  posting_collection_controller_->InsertPosting(posting);
}

inline void IndexingParserCallback::ProcessDocLength(int doc_length, uint32_t doc_id) {
  //will dump to the index.dmap_basic
  posting_collection_controller_->SaveDocLength(doc_length, doc_id);
}

inline void IndexingParserCallback::ProcessUrl(const char* url, int url_len, uint32_t doc_id) {
  // TODO: Wei: 2012/10/02 updated. There are some cases even the url_len is 0, this function
  // still have been called(This is a bug for the clueweb09 dataset). But I just handle this
  // situation here (ideally, should be handled in the indexing and parsing site).

  // This part is for debugging purposes.
  /*
  std::cout << "url_len:" << url_len << std::endl;
  for(int i = 0; i < url_len; i++){
	  std::cout << url[i];
  }
  std::cout << std::endl;
  */

  if (url_len == 0){
	  // Just ingore this branch of logic.
	  // Do not do anything currently.
  }
  else{
	  assert(url_len > 7);  // We strip away the 'http://' part.
	  //will dump to the index.dmap_extended
	  posting_collection_controller_->SaveDocUrl(url + 7, url_len - 7, doc_id);
  }
}

inline void IndexingParserCallback::ProcessDocno(const char* docno, int docno_len, uint32_t doc_id) {
  //will dump to the index.dmap_extended
  posting_collection_controller_->SaveDocno(docno, docno_len, doc_id);
}

/**************************************************************************************************************************************************************
 * DocUrlRetrievalParserCallback
 *
 **************************************************************************************************************************************************************/
class DocUrlRetrievalParserCallback : public ParserCallback {
public:
  void ProcessUrl(const char* url, int url_len, uint32_t doc_id);

  std::vector<std::pair<std::string, uint32_t> >& document_urls() {
    return document_urls_;
  }

private:
  std::vector<std::pair<std::string, uint32_t> > document_urls_;
};

inline void DocUrlRetrievalParserCallback::ProcessUrl(const char* url, int url_len, uint32_t doc_id) {
  const char kCommonUrlPrefix[] = "http://";
  const int kCommonUrlPrefixLen = sizeof(kCommonUrlPrefix) - 1;
  document_urls_.push_back(make_pair(std::string(url + kCommonUrlPrefixLen, url_len - kCommonUrlPrefixLen), doc_id));
}

#endif /* PARSER_CALLBACK_H_ */
